#!/usr/bin/groovy
@Library('test-shared-library') _

// Job parameters
properties(
        [
                parameters(
                        [       booleanParam(name: 'wasH2OUpgraded', defaultValue: true, description: "True if H2O was upgraded in this release or not"),
                                booleanParam(name: 'updateChangeLog', defaultValue: true, description: "Update change log"),
                                string(name: 'releaseFor', defaultValue: 'all', description: "For which Spark Major version" +
                                        " do a release. By default, do release for all supported released versions"),
                                booleanParam(name: 'buildConda', defaultValue: true, description: 'Build Conda'),
                                booleanParam(name: 'publishToNexus', defaultValue: true, description: 'Publish to Nexus'),
                                booleanParam(name: 'buildExtendedH2OJars', defaultValue: true, description: 'Build extended H2O Jars'),
                                booleanParam(name: 'publishToS3', defaultValue: true, description: 'Publish to S3'),
                                booleanParam(name: 'updateDocLinks', defaultValue: true, description: 'Update documentation links'),
                                booleanParam(name: 'publishToPiPy', defaultValue: true, description: 'Publish to PiPy'),
                                booleanParam(name: 'publishConda', defaultValue: true, description: 'Publish to Conda'),
                                booleanParam(name: 'releaseOnGithub', defaultValue: true, description: 'Release on Github')
                        ]
                )
        ]
)

//
// Utility methods for the pipeline
//

def getS3Path() {
    return sh(script: "./gradlew -q s3path", returnStdout: true).trim()
}

def getParallelStageDefinition(params){
    return {
        stage ("Spark ${params.spark}") {
            node('master') {
                docker.withRegistry("http://harbor.h2o.ai") {
                    ansiColor('xterm') {
                        timestamps {
                            ws("${env.WORKSPACE}-spark-${params.spark}") {

                                cleanWs()
                                checkout scm
                                withDocker {
                                    sh "git pull"
                                    build()(params)
                                    buildConda()(params)
                                    publishToNexus()(params)
                                    buildExtendedH2OJars()(params)
                                    publishToS3()(params)
                                    updateDocLinks()(params)
                                    publishToPipy()(params)
                                    publishToConda()(params)
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

String getDockerImageVersion() {
    def versionLine = readFile("gradle.properties").split("\n").find() { line -> line.startsWith('dockerImageVersion') }
    return versionLine.split("=")[1]
}

def withDocker(code) {
    def image = 'opsh2oai/sparkling_water_tests:' + getDockerImageVersion()
    retryWithDelay(3, 120, {
        withCredentials([usernamePassword(credentialsId: "harbor.h2o.ai", usernameVariable: 'REGISTRY_USERNAME', passwordVariable: 'REGISTRY_PASSWORD')]) {
            sh "docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD harbor.h2o.ai"
            sh "docker pull harbor.h2o.ai/${image}"
        }
    })
    docker.image(image).inside("--init --privileged --dns 172.16.0.200") {
        code()
    }
}

String getVersion(params) {
    def versionLine = readFile("gradle.properties").split("\n").find() { line -> line.startsWith('version') }
    return "${versionLine.split("=")[1].replace("-SNAPSHOT", "")}-${params.spark}"
}

String getVersionNoSuffix() {
    def versionLine = readFile("gradle.properties").split("\n").find() { line -> line.startsWith('version') }
    return versionLine.split("=")[1].replace("-SNAPSHOT", "")
}

String getNextVersionNoSuffix(params) {
    def majorVersion = getMajorVersion(params)
    def minorVersion = getMinorVersion(params)
    def patchVersion = getPatchVersion(params)
    if (params.wasH2OUpgraded.toBoolean()) {
        return "${majorVersion}.${minorVersion.toInteger() + 1}-1-SNAPSHOT"
    } else {
        return "${majorVersion}.${minorVersion}-${patchVersion.toInteger() + 1}-SNAPSHOT"
    }
}


String getMajorVersion(params) {
    def v = getVersion(params)
    def split = v.split("-")[0].split("\\.")
    return "${split[0]}.${split[1]}.${split[2]}".toString()
}

String getMinorVersion(params) {
    def v = getVersion(params)
    def split = v.split("-")[0].split("\\.")
    return "${split[3]}".toString()
}

String getPatchVersion(params) {
    def v = getVersion(params)
    def split = v.split("-")
    return "${split[1]}".toString()
}

def prepareReleaseNotes(sparkMajorVersions) {
    def versionNoSparkSuffix = getVersionNoSuffix()
    def path = getS3Path()
    def links = sparkMajorVersions.collect{ majorVersion ->
        "   - for Spark ${majorVersion}: `http://h2o-release.s3.amazonaws.com/sparkling-water/spark-${majorVersion}/${path}${versionNoSparkSuffix}-${majorVersion}/index.html <http://h2o-release.s3.amazonaws.com/sparkling-water/spark-${majorVersion}/${path}${versionNoSparkSuffix}-${majorVersion}/index.html>`__"
    }
    withCredentials([file(credentialsId: 'master-id-rsa', variable: 'ID_RSA_PATH'), file(credentialsId: 'master-gitconfig', variable: 'GITCONFIG_PATH'), string(credentialsId: 'h2o-ops-personal-auth-token', variable: 'GITHUB_TOKEN')]) {
        sh """
            # Copy keys
            rm -rf ~/.ssh
            mkdir -p ~/.ssh
            cp \${ID_RSA_PATH} ~/.ssh/id_rsa
            cp \${GITCONFIG_PATH} ~/.gitconfig
            """
        dir("doc/") {
            writeFile file: "release_notes", text: links.join("\n")
            sh "echo >> release_notes"
            sh """
            # Get ID of this release
            jira_version_id=\$(curl --silent "https://0xdata.atlassian.net/rest/api/2/project/SW/versions" | tr '}' '\\n' | grep "\\"name\\":\\"${
                versionNoSparkSuffix
            }\\"" | cut -d'"' -f8)
            # Get the JIRA page (currently, there is no endpoint for release notes)
            release_notes_page="https://0xdata.atlassian.net/secure/ReleaseNote.jspa?projectId=12000&version=\${jira_version_id}"
        
            # Obtain the release notes and process them so they look like we expect
            curl --silent "\$release_notes_page" | sed '/<body>/,/<a name="editarea"><\\/a>/!d;//D' | sed 's/<ul>//' | sed 's/<\\/ul>//' | sed 's/ *<h2>/-  /' | sed 's/<\\/h2>//'  | sed 's/<\\/li>//' | sed "s/ *<li>\\[<a href='https:\\/\\/0xdata.atlassian.net\\/browse\\/SW-\\([0-9]*\\)'>SW-[0-9]*<\\/a>\\]/\\   -  \\`SW-\\1 <https:\\/\\/0xdata.atlassian.net\\/browse\\/SW-\\1>\\`__/" | sed '\$ d' | sed '1d' >> release_notes

            release_date=\$(date +%Y-%m-%d)
            rel_prefix=\$(echo "v${versionNoSparkSuffix} (\$release_date)")
            underscores=\$(head -c \${#rel_prefix} < /dev/zero | tr '\\0' '-')
            downloads_header="Downloads:\n"
        
            # Insert release info
            sed -i "4i \$rel_prefix" CHANGELOG.rst
        
            # Insert the underscores
            sed -i "5i \$underscores" CHANGELOG.rst
        
            # Insert the downloads header
            sed -i "6i \$downloads_header" CHANGELOG.rst
            
            # Insert the release notes
            sed -i "6r release_notes" CHANGELOG.rst
        
            rm -rf release_notes
            
            git add CHANGELOG.rst
            git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/h2oai/sparkling-water.git"
            git commit -m "Release notes for ${versionNoSparkSuffix}"
            git push --set-upstream origin ${BRANCH_NAME}

            # Update MASTER as well
            LAST_COMMIT=`git rev-parse HEAD`
            git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
            git fetch --no-tags
            git checkout master
            git pull
            git checkout -b "master-changelog-${versionNoSparkSuffix}"
            git cherry-pick -n \$LAST_COMMIT
            git commit -m "Update ChangeLog"
            git push --set-upstream origin master-changelog-${versionNoSparkSuffix}

            wget https://github.com/github/hub/releases/download/v2.5.1/hub-linux-amd64-2.5.1.tgz
            tar -xvf hub-linux-amd64-2.5.1.tgz
            ./hub-linux-amd64-2.5.1/bin/hub pull-request -m "Update ChangeLog"
            git checkout ${env.BRANCH_NAME}
            """
        }
    }
}

def updateNexRelVersion(params) {
    withCredentials([file(credentialsId: 'master-id-rsa', variable: 'ID_RSA_PATH'), file(credentialsId: 'master-gitconfig', variable: 'GITCONFIG_PATH'), string(credentialsId: 'h2o-ops-personal-auth-token', variable: 'GITHUB_TOKEN')]) {
        sh """
                        # Copy keys
                        rm -rf ~/.ssh ~/.gitconfig
                        mkdir -p ~/.ssh
                        cp \${ID_RSA_PATH} ~/.ssh/id_rsa
                        cp \${GITCONFIG_PATH} ~/.gitconfig
                        """

        def version = getVersionNoSuffix()
        def nextVersion = getNextVersionNoSuffix(params)
        retryWithDelay(6, 10, {
            sh """
                        git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/h2oai/sparkling-water.git"
                        git pull
                        if git tag --list | grep -q RELEASE-${version}; then git tag -d RELEASE-${version}; fi
                        if git ls-remote --tags origin | grep -q RELEASE-${version}; then git push --delete origin RELEASE-${version}; fi
                        ./gradlew -Prelease.useAutomaticVersion=true -Prelease.releaseVersion=${version} -Prelease.newVersion=${nextVersion} -PdoRelease release -x check
                        """
        })
    }
}

//
// Main entry point to the pipeline and definition of all stages
//

def sparkMajorVersions
node("master") {
    cleanWs()
    checkout scm
    def versionLine = readFile("gradle.properties")
            .split("\n").find() { line -> line.startsWith('supportedSparkVersions') }
    sparkMajorVersions = versionLine.split("=")[1].split(" ")
}


if (params.releaseFor.toString() != "all") {
    sparkMajorVersions = params.releaseFor.split(" ")
}

parallelStages = [:]
sparkMajorVersions.each { version ->
    config = [:]
    params.each { k, v -> config[k] = v }
    config["spark"] = version
    parallelStages["Spark ${version}"] = getParallelStageDefinition(config.clone())
}

updateChangeLogStage(sparkMajorVersions)(params)
parallel(parallelStages)
updateRelVersionStage()(params)

def updateChangeLogStage(sparkMajorVersions) {
    return { params ->
        stage('Update Change Log') {
            if (params.updateChangeLog.toBoolean()) {
                node("master") {
                    docker.withRegistry("http://harbor.h2o.ai") {
                        ansiColor('xterm') {
                            timestamps {
                                withDocker {
                                    prepareReleaseNotes(sparkMajorVersions)
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

def updateRelVersionStage() {
    return { params ->
        stage('Update Next Release Version') {
            if (params.releaseOnGithub.toBoolean()) {
                node("master") {
                    docker.withRegistry("http://harbor.h2o.ai") {
                        ansiColor('xterm') {
                            timestamps {
                                withDocker {
                                    updateNexRelVersion(params)
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

def build() {
    return { params ->
        stage('Build') {
            withCredentials([usernamePassword(credentialsId: "SIGNING_KEY", usernameVariable: 'SIGN_KEY', passwordVariable: 'SIGN_PASSWORD'),
                             file(credentialsId: 'release-secret-key-ring-file', variable: 'RING_FILE_PATH')]) {
                sh """
                    activate_java_8
                    ./gradlew dist -Pspark=${params.spark} -Pversion=${getVersion(params)} -PdoRelease -Psigning.keyId=${SIGN_KEY} -Psigning.secretKeyRingFile=${RING_FILE_PATH} -Psigning.password=
                    """
            }
        }
    }
}

def buildConda() {
    return { params ->
        stage('Build Conda Packages') {
            if (params.buildConda.toBoolean()) {
                dir("py/build/conda") {
                    def pythonVersions = ['2.7', '3.6']
                    for (pyVersion in pythonVersions) {
                        sh """
                           . /envs/h2o_env_python3.6/bin/activate

                           conda build h2o_pysparkling_${params.spark} --output-folder "." --no-anaconda-upload --py ${pyVersion}

                           PACKAGE_PATH=\$(conda build h2o_pysparkling_${params.spark} --py ${pyVersion} --output-folder "." --output | tail -1)
                           CURRENT_PLATFORM=\$(basename \$(dirname \$PACKAGE_PATH))
                           mkdir -p ../../../dist/build/dist/py/conda/\$CURRENT_PLATFORM
                           cp \$PACKAGE_PATH ../../../dist/build/dist/py/conda/\$CURRENT_PLATFORM/

                           conda convert \$PACKAGE_PATH -p linux-32 -o ../../../dist/build/dist/py/conda/
                           conda convert \$PACKAGE_PATH -p linux-64 -o ../../../dist/build/dist/py/conda/
                           conda convert \$PACKAGE_PATH -p win-32 -o ../../../dist/build/dist/py/conda/
                           conda convert \$PACKAGE_PATH -p win-64 -o ../../../dist/build/dist/py/conda/
                           conda convert \$PACKAGE_PATH -p osx-64 -o ../../../dist/build/dist/py/conda/
                           """
                    }
                }
            }
        }
    }
}

def publishToNexus() {
    return { params ->
        stage('Publish to Nexus') {
            if (params.publishToNexus.toBoolean()) {
                withCredentials([usernamePassword(credentialsId: "PUBLIC_NEXUS", usernameVariable: 'NEXUS_USERNAME', passwordVariable: 'NEXUS_PASSWORD')]) {
                    sh "./gradlew -Pspark=${params.spark} -Pversion=${getVersion(params)} -PnexusUsername=${NEXUS_USERNAME} -PnexusPassword=${NEXUS_PASSWORD} publishToNexus -x check"
                }
            }
        }
    }
}

def buildExtendedH2OJars() {
    return { params ->
        stage('Build Extended H2O Jars') {
            if (params.buildExtendedH2OJars.toBoolean()) {
                sh """
                    # Create extended H2O jar for all supported hadoop distributions
                    HADOOP_DISTRIBUTIONS=`./gradlew -Pspark=${params.spark} -Pversion=${getVersion(params)} -Dorg.gradle.internal.launcher.welcomeMessageEnabled=false -q :sparkling-water-assembly-h2o:printHadoopDistributions)`
                    for distro in \${HADOOP_DISTRIBUTIONS}
                    do
                      ./gradlew -Pspark=${params.spark} -Pversion=${getVersion(params)} -PdoExtend extendJar -PdownloadH2O="\${distro}"
                    done
                    # Create extended H2O jar also for the regular h2o (no H2O driver)
                    ./gradlew -Pspark=${params.spark} -Pversion=${getVersion(params)} -PdoExtend extendJar -PdownloadH2O
                    
                    # Copy to directory which is uploaded to S3
                    cp -R assembly-h2o/private/extended dist/build/dist/
                    """
            }
        }
    }
}

def publishToS3() {
    return { params ->
        stage('Publish to S3') {
            if (params.publishToS3.toBoolean()) {
                def version = getVersion(params)
                def path = getS3Path()
                withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'AWS S3 Credentials', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
                    sh """
                        export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
                        export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
                        ~/.local/bin/aws s3 sync dist/build/dist s3://h2o-release/sparkling-water/spark-${params.spark}/${path}${version} --acl public-read

                        echo UPDATE LATEST POINTER
                        echo ${version} > latest
                        echo "<head>" > latest.html
                        echo "<meta http-equiv=\\"refresh\\" content=\\"0; url=${version}/index.html\\" />" >> latest.html
                        echo "</head>" >> latest.html

                        ~/.local/bin/aws s3 cp latest s3://h2o-release/sparkling-water/spark-${params.spark}/${path}latest --acl public-read
                        ~/.local/bin/aws s3 cp latest.html s3://h2o-release/sparkling-water/spark-${params.spark}/${path}latest.html --acl public-read
                        ~/.local/bin/aws s3 cp latest.html s3://h2o-release/sparkling-water/spark-${params.spark}/${path}index.html --acl public-read
                        """
                }
            }
        }
    }
}

def updateDocLinks() {
    return { params ->
        stage('Update Documentation Links') {
            if (params.updateDocLinks.toBoolean()) {
                withCredentials([file(credentialsId: 'master-id-rsa', variable: 'ID_RSA_PATH'), file(credentialsId: 'master-gitconfig', variable: 'GITCONFIG_PATH'), string(credentialsId: 'h2o-ops-personal-auth-token', variable: 'GITHUB_TOKEN'), sshUserPrivateKey(credentialsId: 'h2oOpsGitPrivateKey', keyFileVariable: 'SSH_KEY_GITHUB')]) {

                    sh """
                               # Copy keys
                               rm -rf ~/.ssh ~/.gitconfig
                               mkdir -p ~/.ssh
                               cp \${ID_RSA_PATH} ~/.ssh/id_rsa
                               cp \${GITCONFIG_PATH} ~/.gitconfig

cat <<EOF >>  ~/.ssh/config
Host github.com
   HostName github.com
   User git
   IdentityFile \${SSH_KEY_GITHUB}
   IdentitiesOnly yes
EOF

                                ssh-keyscan github.com >> ~/.ssh/known_hosts
                               """

                    def version = getVersion(params)
                    retryWithDelay(3, 120, {
                        sh """
                                rm -rf docs.h2o.ai
                                git clone git@github.com:h2oai/docs.h2o.ai.git
                                cd docs.h2o.ai/sites-available/
                                sed -i.backup -E 's?        ProxyPass "/sparkling-water/${params.spark}/latest-stable/".*?        ProxyPass "/sparkling-water/${params.spark}/latest-stable/" "http://h2o-release.s3.amazonaws.com/sparkling-water/spark-${params.spark}/${version}/"?' 000-default.conf
                                git add 000-default.conf
                                git commit -m "Update latest version of Sparkling Water for Spark ${params.spark} version to ${version}"
                                git push --set-upstream origin master
                                cd ../..
                            """
                    })
                }
            }
        }
    }
}

def publishToPipy() {
    return { params ->
        stage('Publish to PiPy') {
            if (params.publishToPiPy.toBoolean()) {
                dir("py/build/pkg") {
                    withCredentials([usernamePassword(credentialsId: "pypi-credentials", usernameVariable: 'PIPY_USERNAME', passwordVariable: 'PIPY_PASSWORD')]) {
                        sh """
                            . /envs/h2o_env_python3.6/bin/activate
                            python setup.py sdist
                            twine upload dist/* -u $PIPY_USERNAME -p $PIPY_PASSWORD
                            """
                    }
                }
            }
        }
    }
}

def publishCondaArtifact(arch, pkgName) {
    retryWithDelay(3, 120, {
        sh "yes | anaconda login --username ${ANACONDA_USERNAME} --password ${ANACONDA_PASSWORD}"
        sh "anaconda upload --force ../../../dist/build/dist/py/conda/${arch}/${pkgName}"
    })
}


def getCondaPkgName(params, pyVersion) {
    return sh(returnStdout: true, script:
            """
            CONDA_PKG_CURRENT_ARCH_PATH=\$(conda build h2o_pysparkling_${params.spark} --py ${pyVersion} --output-folder "." --output | tail -1)
            basename \$CONDA_PKG_CURRENT_ARCH_PATH
            """).trim()
}

def publishToConda() {
    return { params ->
        stage('Publish to Conda') {
            if (params.buildConda.toBoolean() && params.publishConda.toBoolean()) {
                dir("py/build/conda") {
                    withCredentials([usernamePassword(credentialsId: 'anaconda-credentials', usernameVariable: 'ANACONDA_USERNAME', passwordVariable: 'ANACONDA_PASSWORD')]) {
                        def pythonVersions = ['2.7', '3.6']
                        for (pyVersion in pythonVersions) {
                            def pkgName = getCondaPkgName(params, pyVersion)
                            for (arch in ['osx-64', 'linux-32', 'linux-64', 'win-32', 'win-64']) {
                                publishCondaArtifact(arch, pkgName)
                            }
                        }
                    }
                }
            }
        }
    }
}
